{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gc\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.feature_selection import RFE\n",
    "import lightgbm as lgb\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stacked = pd.read_csv('../oofs/kain-train-features-v0.1.2.csv', index_col=0)\n",
    "test_stacked = pd.read_csv('../oofs/kain-test-features-v0.1.2.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "144"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('../../data/application_train.csv.zip', nrows=None)\n",
    "n_train = train.shape[0]\n",
    "y = train['TARGET']\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = lgb.LGBMClassifier(boosting_type='goss', learning_rate=0.03, objective='binary',\n",
    "                         num_leaves=16, subsample=0.8, nthread=4,\n",
    "                         max_depth=4, class_weight={0:1, 1:3}, metric='auc',\n",
    "                         colsample_bytree=0.35, reg_alpha=0, reg_lambda=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "selector = RFE(clf, 45, step=2)\n",
    "selector = selector.fit(train_stacked, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features = [i for i, y in enumerate(selector.ranking_) if y == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[9,\n",
       " 11,\n",
       " 12,\n",
       " 13,\n",
       " 16,\n",
       " 23,\n",
       " 24,\n",
       " 25,\n",
       " 26,\n",
       " 27,\n",
       " 34,\n",
       " 38,\n",
       " 39,\n",
       " 40,\n",
       " 41,\n",
       " 43,\n",
       " 44,\n",
       " 48,\n",
       " 52,\n",
       " 53,\n",
       " 56,\n",
       " 57,\n",
       " 61,\n",
       " 62,\n",
       " 64,\n",
       " 66,\n",
       " 67,\n",
       " 69,\n",
       " 72,\n",
       " 76,\n",
       " 81,\n",
       " 83,\n",
       " 84,\n",
       " 86,\n",
       " 87,\n",
       " 105,\n",
       " 106,\n",
       " 110,\n",
       " 127,\n",
       " 129,\n",
       " 134,\n",
       " 145,\n",
       " 146,\n",
       " 149,\n",
       " 150]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features = pd.DataFrame(train_stacked.iloc[:, selected_features].values, columns=\n",
    "              ['y_' + str(i) for i in selected_features])\n",
    "test_features = pd.DataFrame(test_stacked.iloc[:, selected_features].values, columns=['y_' + str(i) for i in selected_features] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_9</th>\n",
       "      <th>y_11</th>\n",
       "      <th>y_12</th>\n",
       "      <th>y_13</th>\n",
       "      <th>y_16</th>\n",
       "      <th>y_23</th>\n",
       "      <th>y_24</th>\n",
       "      <th>y_25</th>\n",
       "      <th>y_26</th>\n",
       "      <th>y_27</th>\n",
       "      <th>...</th>\n",
       "      <th>y_105</th>\n",
       "      <th>y_106</th>\n",
       "      <th>y_110</th>\n",
       "      <th>y_127</th>\n",
       "      <th>y_129</th>\n",
       "      <th>y_134</th>\n",
       "      <th>y_145</th>\n",
       "      <th>y_146</th>\n",
       "      <th>y_149</th>\n",
       "      <th>y_150</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.027859</td>\n",
       "      <td>0.034761</td>\n",
       "      <td>0.026316</td>\n",
       "      <td>0.027343</td>\n",
       "      <td>0.026407</td>\n",
       "      <td>0.029955</td>\n",
       "      <td>0.031383</td>\n",
       "      <td>0.029491</td>\n",
       "      <td>0.028048</td>\n",
       "      <td>0.030111</td>\n",
       "      <td>...</td>\n",
       "      <td>0.307194</td>\n",
       "      <td>0.092173</td>\n",
       "      <td>0.205684</td>\n",
       "      <td>0.040700</td>\n",
       "      <td>0.079934</td>\n",
       "      <td>0.007814</td>\n",
       "      <td>0.085077</td>\n",
       "      <td>0.059713</td>\n",
       "      <td>0.112115</td>\n",
       "      <td>0.059259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.112407</td>\n",
       "      <td>0.117645</td>\n",
       "      <td>0.123170</td>\n",
       "      <td>0.122424</td>\n",
       "      <td>0.111397</td>\n",
       "      <td>0.122372</td>\n",
       "      <td>0.121970</td>\n",
       "      <td>0.108754</td>\n",
       "      <td>0.106518</td>\n",
       "      <td>0.114218</td>\n",
       "      <td>...</td>\n",
       "      <td>0.735932</td>\n",
       "      <td>0.294385</td>\n",
       "      <td>0.467233</td>\n",
       "      <td>0.136157</td>\n",
       "      <td>0.253435</td>\n",
       "      <td>0.095759</td>\n",
       "      <td>0.236243</td>\n",
       "      <td>0.293807</td>\n",
       "      <td>0.220960</td>\n",
       "      <td>0.127498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.025666</td>\n",
       "      <td>0.021324</td>\n",
       "      <td>0.021003</td>\n",
       "      <td>0.025299</td>\n",
       "      <td>0.023309</td>\n",
       "      <td>0.031361</td>\n",
       "      <td>0.029568</td>\n",
       "      <td>0.028430</td>\n",
       "      <td>0.029473</td>\n",
       "      <td>0.033136</td>\n",
       "      <td>...</td>\n",
       "      <td>0.376777</td>\n",
       "      <td>0.061269</td>\n",
       "      <td>0.088654</td>\n",
       "      <td>0.034251</td>\n",
       "      <td>0.073055</td>\n",
       "      <td>0.027800</td>\n",
       "      <td>0.042244</td>\n",
       "      <td>0.063244</td>\n",
       "      <td>0.016997</td>\n",
       "      <td>0.037535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.020858</td>\n",
       "      <td>0.027794</td>\n",
       "      <td>0.023603</td>\n",
       "      <td>0.024004</td>\n",
       "      <td>0.026581</td>\n",
       "      <td>0.033679</td>\n",
       "      <td>0.030641</td>\n",
       "      <td>0.032629</td>\n",
       "      <td>0.029063</td>\n",
       "      <td>0.032480</td>\n",
       "      <td>...</td>\n",
       "      <td>0.392371</td>\n",
       "      <td>0.105019</td>\n",
       "      <td>0.127739</td>\n",
       "      <td>0.044724</td>\n",
       "      <td>0.077136</td>\n",
       "      <td>0.026577</td>\n",
       "      <td>0.080953</td>\n",
       "      <td>0.120677</td>\n",
       "      <td>0.043664</td>\n",
       "      <td>0.035744</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.114807</td>\n",
       "      <td>0.118162</td>\n",
       "      <td>0.109459</td>\n",
       "      <td>0.111463</td>\n",
       "      <td>0.109929</td>\n",
       "      <td>0.105033</td>\n",
       "      <td>0.111265</td>\n",
       "      <td>0.113932</td>\n",
       "      <td>0.114818</td>\n",
       "      <td>0.106250</td>\n",
       "      <td>...</td>\n",
       "      <td>0.806458</td>\n",
       "      <td>0.284324</td>\n",
       "      <td>0.499732</td>\n",
       "      <td>0.126538</td>\n",
       "      <td>0.284348</td>\n",
       "      <td>0.107383</td>\n",
       "      <td>0.332027</td>\n",
       "      <td>0.250013</td>\n",
       "      <td>0.136448</td>\n",
       "      <td>0.118943</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 45 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        y_9      y_11      y_12      y_13      y_16      y_23      y_24  \\\n",
       "0  0.027859  0.034761  0.026316  0.027343  0.026407  0.029955  0.031383   \n",
       "1  0.112407  0.117645  0.123170  0.122424  0.111397  0.122372  0.121970   \n",
       "2  0.025666  0.021324  0.021003  0.025299  0.023309  0.031361  0.029568   \n",
       "3  0.020858  0.027794  0.023603  0.024004  0.026581  0.033679  0.030641   \n",
       "4  0.114807  0.118162  0.109459  0.111463  0.109929  0.105033  0.111265   \n",
       "\n",
       "       y_25      y_26      y_27    ...        y_105     y_106     y_110  \\\n",
       "0  0.029491  0.028048  0.030111    ...     0.307194  0.092173  0.205684   \n",
       "1  0.108754  0.106518  0.114218    ...     0.735932  0.294385  0.467233   \n",
       "2  0.028430  0.029473  0.033136    ...     0.376777  0.061269  0.088654   \n",
       "3  0.032629  0.029063  0.032480    ...     0.392371  0.105019  0.127739   \n",
       "4  0.113932  0.114818  0.106250    ...     0.806458  0.284324  0.499732   \n",
       "\n",
       "      y_127     y_129     y_134     y_145     y_146     y_149     y_150  \n",
       "0  0.040700  0.079934  0.007814  0.085077  0.059713  0.112115  0.059259  \n",
       "1  0.136157  0.253435  0.095759  0.236243  0.293807  0.220960  0.127498  \n",
       "2  0.034251  0.073055  0.027800  0.042244  0.063244  0.016997  0.037535  \n",
       "3  0.044724  0.077136  0.026577  0.080953  0.120677  0.043664  0.035744  \n",
       "4  0.126538  0.284348  0.107383  0.332027  0.250013  0.136448  0.118943  \n",
       "\n",
       "[5 rows x 45 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN:  [     0      1      2 ... 307505 307506 307508] TEST:  [     9     16     25 ... 307507 307509 307510]\n",
      "1   Fold AUC : 0.8032532917979529\n",
      "2   Fold AUC : 0.8031139063057514\n",
      "3   Fold AUC : 0.8030156154306244\n",
      "4   Fold AUC : 0.803177568336961\n",
      "5   Fold AUC : 0.8032759847959174\n",
      "Fold AUC : 0.8033467369313215\n",
      "TRAIN:  [     0      2      3 ... 307508 307509 307510] TEST:  [     1      8     17 ... 307497 307503 307504]\n",
      "1   Fold AUC : 0.8086677173810604\n",
      "2   Fold AUC : 0.8085551858579718\n",
      "3   Fold AUC : 0.8084422617684013\n",
      "4   Fold AUC : 0.8082541323852884\n",
      "5   Fold AUC : 0.8084206147882113\n",
      "Fold AUC : 0.8086535921907593\n",
      "TRAIN:  [     0      1      4 ... 307507 307509 307510] TEST:  [     2      3      6 ... 307494 307501 307508]\n",
      "1   Fold AUC : 0.8069431543203686\n",
      "2   Fold AUC : 0.8066535704118117\n",
      "3   Fold AUC : 0.8069527604493103\n",
      "4   Fold AUC : 0.8069806436947128\n",
      "5   Fold AUC : 0.8068374299029122\n",
      "Fold AUC : 0.8070061881959467\n",
      "TRAIN:  [     0      1      2 ... 307508 307509 307510] TEST:  [     7     11     13 ... 307495 307498 307506]\n",
      "1   Fold AUC : 0.8010328035364099\n",
      "2   Fold AUC : 0.8009246270646068\n",
      "3   Fold AUC : 0.8008632902594455\n",
      "4   Fold AUC : 0.8006840709976781\n",
      "5   Fold AUC : 0.8009258477064628\n",
      "Fold AUC : 0.8010731044343423\n",
      "TRAIN:  [     1      2      3 ... 307508 307509 307510] TEST:  [     0      4      5 ... 307499 307502 307505]\n",
      "1   Fold AUC : 0.8076687443109047\n",
      "2   Fold AUC : 0.807721287598551\n",
      "3   Fold AUC : 0.8076495214538164\n",
      "4   Fold AUC : 0.8071853345861495\n",
      "5   Fold AUC : 0.8072910889961585\n",
      "Fold AUC : 0.8077152127374446\n",
      "AVERAGED AUC : 0.8055589668979628\n"
     ]
    }
   ],
   "source": [
    "aucs = []\n",
    "test_set = []\n",
    "oof_preds = np.zeros(train.shape[0])\n",
    "\n",
    "kf = KFold(n_splits=5, random_state=1002, shuffle=True)\n",
    "kf.get_n_splits(train_features)\n",
    "\n",
    "n_bagged = 6\n",
    "\n",
    "for train_index, test_index in kf.split(train_features):\n",
    "    print(\"TRAIN: \", train_index, \"TEST: \", test_index)\n",
    "    \n",
    "    X = train_features\n",
    "    y_ = y.values\n",
    "    x_train, x_test = X.iloc[train_index], X.iloc[test_index]\n",
    "    y_train, y_test = y_[train_index], y_[test_index]\n",
    "    \n",
    "    \n",
    "    oof_baggs = np.zeros([n_bagged , x_test.shape[0]])\n",
    "    preds_baggs = np.zeros([n_bagged , test_features.shape[0]])\n",
    "    \n",
    "    for _it in range(1, n_bagged):\n",
    "        \n",
    "        \n",
    "        dtest = test_features\n",
    "        \n",
    "        dtrain = lgb.Dataset(data=x_train, \n",
    "                                 label=y_train, \n",
    "                                 free_raw_data=False, silent=True)\n",
    "        dvalid = lgb.Dataset(data=x_test,\n",
    "                                 label=y_test, \n",
    "                                 free_raw_data=False, silent=True)\n",
    "\n",
    "\n",
    "        params = {\n",
    "                'objective': 'binary',\n",
    "                'boosting_type': 'goss',\n",
    "                'nthread': 4,\n",
    "                'learning_rate': 0.03   \n",
    "                'num_leaves': 2 ** 4,\n",
    "                'colsample_bytree': 0.35,\n",
    "                'subsample': 0.95,\n",
    "                'max_depth': 4,\n",
    "                'reg_alpha': 0,\n",
    "                'reg_lambda': 0,\n",
    "                'seed': _it,\n",
    "                'scale_pos_weight': 3,\n",
    "                'verbose': -1,\n",
    "                'metric': 'auc'\n",
    "            }\n",
    "        \n",
    "        \n",
    "        \n",
    "        model = lgb.train(\n",
    "                params=params,\n",
    "                train_set=dtrain,\n",
    "                num_boost_round=10000,\n",
    "                valid_sets=[dtrain, dvalid],\n",
    "                early_stopping_rounds=100,\n",
    "                verbose_eval=False\n",
    "            ) \n",
    "        print(_it,' ' ,'Fold AUC :', roc_auc_score(y_test, model.predict(x_test)))\n",
    "        oof_baggs[_it, :] = model.predict(x_test)\n",
    "        preds_baggs[_it, :] = model.predict(dtest)\n",
    "        \n",
    "    val_preds = pd.DataFrame(oof_baggs).T\n",
    "    test_preds = pd.DataFrame(preds_baggs).T\n",
    "    \n",
    "    oof_preds[test_index] = val_preds.rank(axis=0, method='min').mul(val_preds.shape[1] * [1 / val_preds.shape[1]]).sum(1) / val_preds.shape[0]\n",
    "\n",
    "    print('Fold AUC :', roc_auc_score(y_test, val_preds.rank(axis=0, method='min').mul(val_preds.shape[1] * [1 / val_preds.shape[1]]).sum(1) / val_preds.shape[0]))\n",
    "    aucs.append(roc_auc_score(y_test, val_preds.rank(axis=0, method='min').mul(val_preds.shape[1] * [1 / val_preds.shape[1]]).sum(1) / val_preds.shape[0]))\n",
    "   \n",
    "\n",
    "    test_set.append(test_preds.rank(axis=0, method='min').mul(test_preds.shape[1] * [1 / test_preds.shape[1]]).sum(1) / test_preds.shape[0])\n",
    "    gc.collect()\n",
    "    \n",
    "\n",
    "print('AVERAGED AUC :', np.mean(aucs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = pd.DataFrame(test_set).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100001</td>\n",
       "      <td>0.387863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100005</td>\n",
       "      <td>0.803968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100013</td>\n",
       "      <td>0.369838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100028</td>\n",
       "      <td>0.383772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100038</td>\n",
       "      <td>0.814689</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR    TARGET\n",
       "0      100001  0.387863\n",
       "1      100005  0.803968\n",
       "2      100013  0.369838\n",
       "3      100028  0.383772\n",
       "4      100038  0.814689"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_hat = preds.rank(axis=0, method='min').mul(preds.shape[1] * [1 / preds.shape[1]]).sum(1) / preds.shape[0] \n",
    "    \n",
    "\n",
    "\n",
    "sampl_sub = pd.read_csv('../../data/sample_submission.csv')\n",
    "\n",
    "\n",
    "sampl_sub['TARGET'] = y_hat.values\n",
    "\n",
    "sampl_sub.to_csv(\"lightgbm-stack-submission.csv\", index=False)\n",
    "\n",
    "\n",
    "sampl_sub.head()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
